package com.stoney.handler;

import com.stoney.cache.DefaultSpiderCache;
import com.stoney.cache.SpiderCache;

import com.stoney.utils.FilesUtil;
import com.stoney.utils.UrlUtil;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.regex.Matcher;
import java.util.regex.Pattern;



/**
 * Created by Stony on 2015/11/24.
 */
public abstract class SpiderAbstractHandler implements SpiderBaseHandler {

    private static final int OPERATION_TIMEOUT = 60 * 1000;
    public static final String regex = "(url[(]{1}[\"']?)([./a-zA-Z0-9-_?=#&:]+)(['\"]?[)]{1})";

    private final static  org.apache.log4j.Logger logger = org.apache.log4j.Logger.getLogger(SpiderAbstractHandler.class);
    public void print(String format,Object...agrs){
        logger.info(String.format(format,agrs));
    }

    /**
     * 1)add(anObject):把anObject加到BlockingQueue里,即如果BlockingQueue可以容纳,则返回true,否则招聘异常
     2)offer(anObject):表示如果可能的话,将anObject加到BlockingQueue里,即如果BlockingQueue可以容纳,则返回true,否则返回false.
     3)put(anObject):把anObject加到BlockingQueue里,如果BlockQueue没有空间,则调用此方法的线程被阻断直到BlockingQueue里面有空间再继续.
     4)poll(time):取走BlockingQueue里排在首位的对象,若不能立即取出,则可以等time参数规定的时间,取不到时返回null
     5)take():取走BlockingQueue里排在首位的对象,若BlockingQueue为空,阻断进入等待状态直到Blocking有新的对象被加入为止
     */
    public ArrayBlockingQueue mediaQueue;
    public ArrayBlockingQueue repeatQueue;
    public ArrayBlockingQueue cssQueue;
    public ArrayBlockingQueue pageQueue;
    public List<SpiderResource.SpiderResourceType> ignore;
    public ArrayBlockingQueue<SpiderResource> spiderResourceQueue;
    public SpiderCache cache;

    protected ExecutorService service;
    protected boolean isInitQueue = true;
    protected boolean isInitConfig = true;
    protected boolean isInitExecutorService = true;
    protected SpiderAbstractHandler() {
        this(true,true,true);
    }
    protected SpiderAbstractHandler(boolean isInitQueue, boolean isInitConfig, boolean isInitExecutorService) {
        this.isInitQueue = isInitQueue;
        this.isInitConfig = isInitConfig;
        this.isInitExecutorService = isInitExecutorService;
        init();
    }
    public void init() {
        if(isInitQueue){
            initQueue();
        }
        if(isInitExecutorService){
            setExecutorService(Executors.newFixedThreadPool(8));//Executors.newCachedThreadPool()); //Executors.newFixedThreadPool(8);
        }
        if(isInitConfig){
           initConfig();
        }
    }
    public void setExecutorService(ExecutorService executorService){
        this.service = executorService;
    }
    public abstract void initQueue();
    public abstract void initConfig();
    public abstract void run();
    @Override
    public void process(String url) {
        parsePage(url);
        startWork();
    }
    protected abstract void startWork();
    public void setCache(SpiderCache cache) {
        this.cache = cache;
    }

    public SpiderCache getCache() {
        if(cache == null){
            cache = new DefaultSpiderCache();
        }
        return cache;
    }

    public void putPage(String url,boolean isMatcher){
        if(pageQueue != null){
            if(isMathUrl(url)) putPage(url);
        }
    }
    public void putPage(String url){
        if(pageQueue == null) return;
        try {
            pageQueue.offer(url);
        } catch (Exception e) {
        }
    }
    public void putResource(String url,SpiderResource.SpiderResourceType type){
        if(spiderResourceQueue != null){
            putResource(new SpiderResource(url,type));
        }
    }
    public void putResource(SpiderResource resource){
        if(spiderResourceQueue == null) return;
        try {
            spiderResourceQueue.offer(resource);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    public void putMedia(String url){
        if(mediaQueue == null) return;
        try {
            mediaQueue.offer(url);
        } catch (Exception e) {
        }
    }
    public void putCss(String url){
        if(cssQueue == null) return;
        try {
            cssQueue.offer(url);
        } catch (Exception e) {
        }
    }
    public void putRepeat(String url){
        if(mediaQueue == null) return;
        try {
            mediaQueue.offer(url);
        } catch (Exception e) {
        }
    }

    public void parsePage(String url){
        if(FilesUtil.isEmpty(url)) return;
        Document doc = null;
        print(" * enter parsePage <%s>", url);
        try {
            //header("Cookie", COOKIES)
            doc = Jsoup.connect(url).timeout(OPERATION_TIMEOUT).get();

            Elements links = doc.select("a[href]");
            Elements media = doc.select("[src]");
            Elements imports = doc.select("link[href]");
            print("\nMedia: (%d)", media.size());
            for (Element src : media) {
                if (src.tagName().equals("img")){
                    putMedia(src.attr("abs:src"));
                    putResource(src.attr("abs:src"), SpiderResource.SpiderResourceType.MEDIA);
                    print(" * %s: <%s> %sx%s (%s)",src.tagName(), src.attr("abs:src"), src.attr("width"), src.attr("height"),trim(src.attr("alt"), 20));
                }else{
                    putMedia(src.attr("abs:src"));
                    putResource(src.attr("abs:src"), SpiderResource.SpiderResourceType.MEDIA);
                    print(" * %s: <%s>", src.tagName(), src.attr("abs:src"));
                }
            }
            print("\nCss Imports: (%d)", imports.size());
            for (Element link : imports) {
                String tagName = link.tagName();
                if(tagName.endsWith(".png") || tagName.endsWith(".gif") || tagName.endsWith(".jpg") || tagName.endsWith(".ico")){
                    putCss(link.attr("abs:href"));
                    putResource(link.attr("abs:href"), SpiderResource.SpiderResourceType.MEDIA);
                    print(" * %s: <%s>", tagName, link.attr("abs:href"));
                }else{
                    putCss(link.attr("abs:href"));
                    putResource(link.attr("abs:href"), SpiderResource.SpiderResourceType.CSS);
                    print(" * %s <%s> (%s)", tagName,link.attr("abs:href"), link.attr("rel"));
                }
            }
            print("\nA Links: (%d)", links.size());
            for (Element link : links) {
                if("a".equals(link.tagName())){
                    print(" * a: <%s>  (%s)", link.attr("abs:href"), trim(link.text(), 35));
                    putPage(link.attr("abs:href"));
                    putResource(link.attr("abs:href"), SpiderResource.SpiderResourceType.HTML);
                }else{
                    print(" * %s: <%s>", link.tagName(), link.attr("abs:href"));
                    putResource(link.attr("abs:href"), SpiderResource.SpiderResourceType.HTML);
                    putPage(link.attr("abs:href"));
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
            putRepeat(url);
        } finally {
            if(doc != null){
                //------- save page
                savePage(url, doc.html());
            }
        }
    }
    public void parseMedia(String url){
        if(FilesUtil.isEmpty(url)) return;
        print(" * enter parseMedia <%s>", url);
        if(isCss(url)){
            parseCss(url);
        }else if(isJs(url)){
            parseJs(url);
        }else {
            saveResource(url);
        }
    }
    public void parseJs(String url){
        if(FilesUtil.isEmpty(url)) return;
        try {
            print(" * enter parseJs <%s>", url);
//            String html = FilesUtil.getHtml(url);
//            Pattern regex = Pattern.compile("([(]?[\"']{1})([./a-zA-Z0-9-_?=#&:]+)(['\"]{1}[)]?)");
//            Matcher mat = regex.matcher(html);
//            while(mat.find()){
//                String src = mat.group(2);
//                if(!isAbsUrl(src)){
//                    src = getBaseUrl() + src;
//                }
//                print(" * %s: <%s>", "find", src);
//                if(isJs(src)){
//                    putMedia(src);
//                }
//                else if(isHtml(src)){
//                    putPage(src);
//                }
//                else if(isCss(src)){
//                    putCss(src);
//                }
//            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            saveResource(url);
        }
    }
    public void parseCss(String url){
        if(FilesUtil.isEmpty(url)) return;
        if(checkNotCss(url)) return;
        print(" * enter parseCss <%s>", url);
        Document doc = null;
        try {
            doc = Jsoup.connect(url).timeout(OPERATION_TIMEOUT).get();
            String baseUri = doc.baseUri();
            Pattern pat = Pattern.compile(regex);
            Matcher mat = pat.matcher(doc.text());
            while(mat.find()){
                String src = mat.group(2);
                if(!isAbsUrl(src)){
                    src = UrlUtil.absUrl(baseUri, src);
                }
                print(" * %s: <%s>", "find", src);
                putMedia(src);
            }
        } catch (Exception e) {
            e.printStackTrace();
            putRepeat(url);
        } finally {
            saveResource(url);
        }
    }
    public boolean isAbsUrl(String url){
        return url.startsWith("http:") || url.startsWith("https:");
    }
    public boolean checkStr(String src, String index){
        return src.contains(index) || src.matches(index);
    }
    public boolean isCss(String url){
        if(url.endsWith(".css") || checkStr(url, ".css")){
            return true;
        }
        return false;
    }
    public boolean isJs(String url){
        if(url.endsWith(".js") || checkStr(url, ".js")){
            return true;
        }
        return false;
    }
    public boolean isHtml(String url){
        if(url.endsWith(".html") || checkStr(url, ".html")){
            return true;
        }
        return false;
    }
    /**
     *
     * @param url
     * @return if it is not css file return true else return false
     */
    public boolean checkNotCss(String url){
        if(url.endsWith(".png") || url.endsWith(".gif") || url.endsWith(".jpg") || url.endsWith(".ico")
                || url.endsWith(".js") || url.endsWith(".html")){
            print(" * %s: <%s>", "img", url);
            return true;
        }
        if(url.endsWith(".ttf") || url.endsWith(".woff") || url.endsWith(".eot") || url.endsWith(".svg")){
            print(" * %s: <%s>", "font", url);
            return true;
        }
        if(checkStr(url,".png") || checkStr(url,".gif") || checkStr(url,".jpg") || checkStr(url,".ico")
                || checkStr(url,".js") || checkStr(url,".html")){
            print(" * %s: <%s>", "img", url);
            return true;
        }
        if(checkStr(url, ".ttf") || checkStr(url, ".woff") || checkStr(url, ".eot") || checkStr(url, ".svg")){
            print(" * %s: <%s>", "font", url);
            return true;
        }
        return false;
    }

    public void saveCache(String url){
        getCache().saveValue(url);
    }

    public abstract String getSaveDir();
    public abstract String getMathUrl();
    public abstract String getBaseUrl();

    public boolean containsCache(String url){
        return cache.contains(url);
    }
    public void saveImg(String url){
        FilesUtil.saveImg(url,getSaveDir());

    }
    public void savePage(String url){
        if(containsCache(url)) return;
        if (FilesUtil.savePage(url, getSaveDir())) {
            saveCache(url);
        }else{
            putRepeat(url);
        }
    }
    public void savePage(String url,String html){
        if(containsCache(url)) return;
        if(FilesUtil.savePage(url, getSaveDir(),html)){
            saveCache(url);
        }else{
            putRepeat(url);
        }
    }
    public void saveResource(String url){
        if(containsCache(url)) return;
        if (FilesUtil.saveResource(url, getSaveDir())) {
            saveCache(url);
        }else{
            putRepeat(url);
        }
    }
    public boolean isMathUrl(String url){
        if(FilesUtil.isEmpty(getMathUrl())){
            return true;
        }
        return url.contains(getMathUrl()) || url.matches(getMathUrl());
    }
    public String trim(String s, int width) {
        if (s.length() > width)
            return s.substring(0, width - 1) + ".";
        else
            return s;
    }
    public String getHref(Element link, String prefix) {
        String href;
        if (FilesUtil.isEmpty(link.attr("abs:href"))) {
            href = link.attr("href");
        } else {
            href = link.attr("abs:href");
        }
        if (!href.startsWith("http:")) {
            href = prefix + href;
        }
        return href;
    }

    public String getHref(Element link) {
        return getHref(link, "");
    }

}
